home *** CD-ROM | disk | FTP | other *** search
/ Enter 2006 September / Enter 09 2006.iso / Internet / SpamExperts Home 1.1 / SpamExperts Home.exe / lib / spamexperts.modules / email / FeedParser.pyc (.txt) < prev    next >
Encoding:
Python Compiled Bytecode  |  2006-07-14  |  10.4 KB  |  448 lines
  1. # Source Generated with Decompyle++
  2. # File: in.pyc (Python 2.4)
  3.  
  4. """FeedParser - An email feed parser.
  5.  
  6. The feed parser implements an interface for incrementally parsing an email
  7. message, line by line.  This has advantages for certain applications, such as
  8. those reading email messages off a socket.
  9.  
  10. FeedParser.feed() is the primary interface for pushing new data into the
  11. parser.  It returns when there's nothing more it can do with the available
  12. data.  When you have no more data to push into the parser, call .close().
  13. This completes the parsing and returns the root message object.
  14.  
  15. The other advantage of this parser is that it will never throw a parsing
  16. exception.  Instead, when it finds something unexpected, it adds a 'defect' to
  17. the current message.  Defects are just instances that live on the message
  18. object's .defects attribute.
  19. """
  20. import re
  21. from email import Errors
  22. from email import Message
  23. NLCRE = re.compile('\r\n|\r|\n')
  24. NLCRE_bol = re.compile('(\r\n|\r|\n)')
  25. NLCRE_eol = re.compile('(\r\n|\r|\n)$')
  26. NLCRE_crack = re.compile('(\r\n|\r|\n)')
  27. headerRE = re.compile('^(From |[\\041-\\071\\073-\\176]{1,}:|[\\t ])')
  28. EMPTYSTRING = ''
  29. NL = '\n'
  30. NeedMoreData = object()
  31.  
  32. class BufferedSubFile(object):
  33.     '''A file-ish object that can have new data loaded into it.
  34.  
  35.     You can also push and pop line-matching predicates onto a stack.  When the
  36.     current predicate matches the current line, a false EOF response
  37.     (i.e. empty string) is returned instead.  This lets the parser adhere to a
  38.     simple abstraction -- it parses until EOF closes the current message.
  39.     '''
  40.     
  41.     def __init__(self):
  42.         self._partial = ''
  43.         self._lines = []
  44.         self._eofstack = []
  45.         self._closed = False
  46.  
  47.     
  48.     def push_eof_matcher(self, pred):
  49.         self._eofstack.append(pred)
  50.  
  51.     
  52.     def pop_eof_matcher(self):
  53.         return self._eofstack.pop()
  54.  
  55.     
  56.     def close(self):
  57.         self._lines.append(self._partial)
  58.         self._partial = ''
  59.         self._closed = True
  60.  
  61.     
  62.     def readline(self):
  63.         if not self._lines:
  64.             if self._closed:
  65.                 return ''
  66.             
  67.             return NeedMoreData
  68.         
  69.         line = self._lines.pop()
  70.         for ateof in self._eofstack[::-1]:
  71.             if ateof(line):
  72.                 self._lines.append(line)
  73.                 return ''
  74.                 continue
  75.         
  76.         return line
  77.  
  78.     
  79.     def unreadline(self, line):
  80.         if not line is not NeedMoreData:
  81.             raise AssertionError
  82.         self._lines.append(line)
  83.  
  84.     
  85.     def push(self, data):
  86.         '''Push some new data into this object.'''
  87.         data = self._partial + data
  88.         self._partial = ''
  89.         parts = NLCRE_crack.split(data)
  90.         self._partial = parts.pop()
  91.         lines = []
  92.         for i in range(len(parts) // 2):
  93.             lines.append(parts[i * 2] + parts[i * 2 + 1])
  94.         
  95.         self.pushlines(lines)
  96.  
  97.     
  98.     def pushlines(self, lines):
  99.         self._lines[:0] = lines[::-1]
  100.  
  101.     
  102.     def is_closed(self):
  103.         return self._closed
  104.  
  105.     
  106.     def __iter__(self):
  107.         return self
  108.  
  109.     
  110.     def next(self):
  111.         line = self.readline()
  112.         if line == '':
  113.             raise StopIteration
  114.         
  115.         return line
  116.  
  117.  
  118.  
  119. class FeedParser:
  120.     '''A feed-style parser of email.'''
  121.     
  122.     def __init__(self, _factory = Message.Message):
  123.         '''_factory is called with no arguments to create a new message obj'''
  124.         self._factory = _factory
  125.         self._input = BufferedSubFile()
  126.         self._msgstack = []
  127.         self._parse = self._parsegen().next
  128.         self._cur = None
  129.         self._last = None
  130.         self._headersonly = False
  131.  
  132.     
  133.     def _set_headersonly(self):
  134.         self._headersonly = True
  135.  
  136.     
  137.     def feed(self, data):
  138.         '''Push more data into the parser.'''
  139.         self._input.push(data)
  140.         self._call_parse()
  141.  
  142.     
  143.     def _call_parse(self):
  144.         
  145.         try:
  146.             self._parse()
  147.         except StopIteration:
  148.             pass
  149.  
  150.  
  151.     
  152.     def close(self):
  153.         '''Parse all remaining data and return the root message object.'''
  154.         self._input.close()
  155.         self._call_parse()
  156.         root = self._pop_message()
  157.         if not not (self._msgstack):
  158.             raise AssertionError
  159.         if root.get_content_maintype() == 'multipart' and not root.is_multipart():
  160.             root.defects.append(Errors.MultipartInvariantViolationDefect())
  161.         
  162.         return root
  163.  
  164.     
  165.     def _new_message(self):
  166.         msg = self._factory()
  167.         if self._cur and self._cur.get_content_type() == 'multipart/digest':
  168.             msg.set_default_type('message/rfc822')
  169.         
  170.         if self._msgstack:
  171.             self._msgstack[-1].attach(msg)
  172.         
  173.         self._msgstack.append(msg)
  174.         self._cur = msg
  175.         self._last = msg
  176.  
  177.     
  178.     def _pop_message(self):
  179.         retval = self._msgstack.pop()
  180.         if self._msgstack:
  181.             self._cur = self._msgstack[-1]
  182.         else:
  183.             self._cur = None
  184.         return retval
  185.  
  186.     
  187.     def _parsegen(self):
  188.         self._new_message()
  189.         headers = []
  190.         for line in self._input:
  191.             if line is NeedMoreData:
  192.                 yield NeedMoreData
  193.                 continue
  194.             
  195.             if not headerRE.match(line):
  196.                 if not NLCRE.match(line):
  197.                     self._input.unreadline(line)
  198.                 
  199.                 break
  200.             
  201.             headers.append(line)
  202.         
  203.         self._parse_headers(headers)
  204.         if self._headersonly:
  205.             lines = []
  206.             while True:
  207.                 line = self._input.readline()
  208.                 if line is NeedMoreData:
  209.                     yield NeedMoreData
  210.                     continue
  211.                 
  212.                 if line == '':
  213.                     break
  214.                 
  215.                 lines.append(line)
  216.             self._cur.set_payload(EMPTYSTRING.join(lines))
  217.             return None
  218.         
  219.         if self._cur.get_content_type() == 'message/delivery-status':
  220.             while True:
  221.                 self._input.push_eof_matcher(NLCRE.match)
  222.                 for retval in self._parsegen():
  223.                     if retval is NeedMoreData:
  224.                         yield NeedMoreData
  225.                         continue
  226.                     
  227.                 
  228.                 msg = self._pop_message()
  229.                 self._input.pop_eof_matcher()
  230.                 while True:
  231.                     line = self._input.readline()
  232.                     if line is NeedMoreData:
  233.                         yield NeedMoreData
  234.                         continue
  235.                     
  236.                     break
  237.                 while True:
  238.                     line = self._input.readline()
  239.                     if line is NeedMoreData:
  240.                         yield NeedMoreData
  241.                         continue
  242.                     
  243.                     break
  244.                 if line == '':
  245.                     break
  246.                 
  247.                 self._input.unreadline(line)
  248.             return None
  249.         
  250.         if self._cur.get_content_maintype() == 'message':
  251.             for retval in self._parsegen():
  252.                 if retval is NeedMoreData:
  253.                     yield NeedMoreData
  254.                     continue
  255.                 
  256.             
  257.             self._pop_message()
  258.             return None
  259.         
  260.         if self._cur.get_content_maintype() == 'multipart':
  261.             boundary = self._cur.get_boundary()
  262.             if boundary is None:
  263.                 self._cur.defects.append(Errors.NoBoundaryInMultipartDefect())
  264.                 lines = []
  265.                 for line in self._input:
  266.                     if line is NeedMoreData:
  267.                         yield NeedMoreData
  268.                         continue
  269.                     
  270.                     lines.append(line)
  271.                 
  272.                 self._cur.set_payload(EMPTYSTRING.join(lines))
  273.                 return None
  274.             
  275.             separator = '--' + boundary
  276.             boundaryre = re.compile('(?P<sep>' + re.escape(separator) + ')(?P<end>--)?(?P<ws>[ \\t]*)(?P<linesep>\\r\\n|\\r|\\n)?$')
  277.             capturing_preamble = True
  278.             preamble = []
  279.             linesep = False
  280.             while True:
  281.                 line = self._input.readline()
  282.                 if line is NeedMoreData:
  283.                     yield NeedMoreData
  284.                     continue
  285.                 
  286.                 if line == '':
  287.                     break
  288.                 
  289.                 mo = boundaryre.match(line)
  290.                 if mo:
  291.                     if mo.group('end'):
  292.                         linesep = mo.group('linesep')
  293.                         break
  294.                     
  295.                     if capturing_preamble:
  296.                         if preamble:
  297.                             lastline = preamble[-1]
  298.                             eolmo = NLCRE_eol.search(lastline)
  299.                             if eolmo:
  300.                                 preamble[-1] = lastline[:-len(eolmo.group(0))]
  301.                             
  302.                             self._cur.preamble = EMPTYSTRING.join(preamble)
  303.                         
  304.                         capturing_preamble = False
  305.                         self._input.unreadline(line)
  306.                         continue
  307.                     
  308.                     while True:
  309.                         line = self._input.readline()
  310.                         if line is NeedMoreData:
  311.                             yield NeedMoreData
  312.                             continue
  313.                         
  314.                         mo = boundaryre.match(line)
  315.                         if not mo:
  316.                             self._input.unreadline(line)
  317.                             break
  318.                             continue
  319.                     self._input.push_eof_matcher(boundaryre.match)
  320.                     for retval in self._parsegen():
  321.                         if retval is NeedMoreData:
  322.                             yield NeedMoreData
  323.                             continue
  324.                         
  325.                     
  326.                     if self._last.get_content_maintype() == 'multipart':
  327.                         epilogue = self._last.epilogue
  328.                         if epilogue == '':
  329.                             self._last.epilogue = None
  330.                         elif epilogue is not None:
  331.                             mo = NLCRE_eol.search(epilogue)
  332.                             if mo:
  333.                                 end = len(mo.group(0))
  334.                                 self._last.epilogue = epilogue[:-end]
  335.                             
  336.                         
  337.                     else:
  338.                         payload = self._last.get_payload()
  339.                         if isinstance(payload, basestring):
  340.                             mo = NLCRE_eol.search(payload)
  341.                             if mo:
  342.                                 payload = payload[:-len(mo.group(0))]
  343.                                 self._last.set_payload(payload)
  344.                             
  345.                         
  346.                     self._input.pop_eof_matcher()
  347.                     self._pop_message()
  348.                     self._last = self._cur
  349.                     continue
  350.                 if not capturing_preamble:
  351.                     raise AssertionError
  352.                 preamble.append(line)
  353.             if capturing_preamble:
  354.                 self._cur.defects.append(Errors.StartBoundaryNotFoundDefect())
  355.                 self._cur.set_payload(EMPTYSTRING.join(preamble))
  356.                 epilogue = []
  357.                 for line in self._input:
  358.                     if line is NeedMoreData:
  359.                         yield NeedMoreData
  360.                         continue
  361.                         continue
  362.                 
  363.                 self._cur.epilogue = EMPTYSTRING.join(epilogue)
  364.                 return None
  365.             
  366.             if linesep:
  367.                 epilogue = [
  368.                     '']
  369.             else:
  370.                 epilogue = []
  371.             for line in self._input:
  372.                 if line is NeedMoreData:
  373.                     yield NeedMoreData
  374.                     continue
  375.                 
  376.                 epilogue.append(line)
  377.             
  378.             if epilogue:
  379.                 firstline = epilogue[0]
  380.                 bolmo = NLCRE_bol.match(firstline)
  381.                 if bolmo:
  382.                     epilogue[0] = firstline[len(bolmo.group(0)):]
  383.                 
  384.             
  385.             self._cur.epilogue = EMPTYSTRING.join(epilogue)
  386.             return None
  387.         
  388.         lines = []
  389.         for line in self._input:
  390.             if line is NeedMoreData:
  391.                 yield NeedMoreData
  392.                 continue
  393.             
  394.             lines.append(line)
  395.         
  396.         self._cur.set_payload(EMPTYSTRING.join(lines))
  397.  
  398.     
  399.     def _parse_headers(self, lines):
  400.         lastheader = ''
  401.         lastvalue = []
  402.         for lineno, line in enumerate(lines):
  403.             if line[0] in ' \t':
  404.                 if not lastheader:
  405.                     defect = Errors.FirstHeaderLineIsContinuationDefect(line)
  406.                     self._cur.defects.append(defect)
  407.                     continue
  408.                 
  409.                 lastvalue.append(line)
  410.                 continue
  411.             
  412.             if lastheader:
  413.                 lhdr = EMPTYSTRING.join(lastvalue)[:-1].rstrip('\r\n')
  414.                 self._cur[lastheader] = lhdr
  415.                 lastheader = ''
  416.                 lastvalue = []
  417.             
  418.             if line.startswith('From '):
  419.                 if lineno == 0:
  420.                     mo = NLCRE_eol.search(line)
  421.                     if mo:
  422.                         line = line[:-len(mo.group(0))]
  423.                     
  424.                     self._cur.set_unixfrom(line)
  425.                     continue
  426.                 elif lineno == len(lines) - 1:
  427.                     self._input.unreadline(line)
  428.                     return None
  429.                 else:
  430.                     defect = Errors.MisplacedEnvelopeHeaderDefect(line)
  431.                     self._cur.defects.append(defect)
  432.             
  433.             i = line.find(':')
  434.             if i < 0:
  435.                 defect = Errors.MalformedHeaderDefect(line)
  436.                 self._cur.defects.append(defect)
  437.                 continue
  438.             
  439.             lastheader = line[:i]
  440.             lastvalue = [
  441.                 line[i + 1:].lstrip()]
  442.         
  443.         if lastheader:
  444.             self._cur[lastheader] = EMPTYSTRING.join(lastvalue).rstrip('\r\n')
  445.         
  446.  
  447.  
  448.